{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context(\"poster\")\n",
    "sns.set_style(\"ticks\")\n",
    "\n",
    "from itertools import chain\n",
    "\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "import sklearn_crfsuite\n",
    "from sklearn_crfsuite import scorers\n",
    "from sklearn_crfsuite import metrics\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "\n",
    "import regex as re\n",
    "from collections import namedtuple, defaultdict, Counter, OrderedDict\n",
    "from IPython.display import display\n",
    "from joblib import load, dump, Parallel, delayed\n",
    "\n",
    "import os, string, sys\n",
    "\n",
    "from gensim.models import word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class RegexFeatures(object):\n",
    "    PATTERNS = {\n",
    "        \"isInitCapitalWord\": re.compile(r'^[A-Z][a-z]+'),\n",
    "        \"isAllCapitalWord\": re.compile(r'^[A-Z][A-Z]+$'),\n",
    "        \"isAllSmallCase\": re.compile(r'^[a-z]+$'),\n",
    "        \"isWord\": re.compile(r'^[a-zA-Z][a-zA-Z]+$'),\n",
    "        \"isAlphaNumeric\": re.compile(r'^\\p{Alnum}+$'),\n",
    "        \"isSingleCapLetter\": re.compile(r'^[A-Z]$'),\n",
    "        \"containsDashes\": re.compile(r'.*--.*'),\n",
    "        \"containsDash\": re.compile(r'.*\\-.*'),\n",
    "        \"singlePunctuation\": re.compile(r'^\\p{Punct}$'),\n",
    "        \"repeatedPunctuation\": re.compile(r'^[\\.\\,!\\?\"\\':;_\\-]{2,}$'),\n",
    "        \"singleDot\": re.compile(r'[.]'),\n",
    "        \"singleComma\": re.compile(r'[,]'),\n",
    "        \"singleQuote\": re.compile(r'[\\']'),\n",
    "        \"isSpecialCharacter\": re.compile(r'^[#;:\\-/<>\\'\\\"()&]$'),\n",
    "        \"fourDigits\": re.compile(r'^\\d\\d\\d\\d$'),\n",
    "        \"isDigits\": re.compile(r'^\\d+$'),\n",
    "        \"isNumber\": re.compile(r'^((\\p{N}{,2}([,]?\\p{N}{3})+)(\\.\\p{N}+)?)$'),\n",
    "        \"containsDigit\": re.compile(r'.*\\d+.*'),\n",
    "        \"endsWithDot\": re.compile(r'\\p{Alnum}+\\.$'),\n",
    "        \"isURL\": re.compile(r'^http[s]?://'),\n",
    "        \"isMention\": re.compile(r'^(RT)?@[\\p{Alnum}_]+$'),\n",
    "        \"isHashtag\": re.compile(r'^#\\p{Alnum}+$'),\n",
    "        \"isMoney\": re.compile(r'^\\$((\\p{N}{,2}([,]?\\p{N}{3})+)(\\.\\p{N}+)?)$'),\n",
    "    }\n",
    "    def __init__(self):\n",
    "        print \"Initialized RegexFeature\"\n",
    "    @staticmethod\n",
    "    def process(word):\n",
    "        features = dict()\n",
    "        for k, p in RegexFeatures.PATTERNS.iteritems():\n",
    "            if p.match(word):\n",
    "                features[k] = True\n",
    "        return features\n",
    "    \n",
    "    \n",
    "def classification_report_to_df(report):\n",
    "    report_list = []\n",
    "    for i, line in enumerate(report.split(\"\\n\")):\n",
    "        if i == 0:\n",
    "            report_list.append([\"class\", \"precision\", \"recall\", \"f1-score\", \"support\"])\n",
    "        else:\n",
    "            line = line.strip()\n",
    "            if line:\n",
    "                if line.startswith(\"avg\"):\n",
    "                    line = line.replace(\"avg / total\", \"avg/total\")\n",
    "                line = re.split(r'\\s+', line)\n",
    "                report_list.append(tuple(line))\n",
    "    return pd.DataFrame(report_list[1:], columns=report_list[0])\n",
    "\n",
    "\n",
    "DATA_DIR=\"data/data/\"\n",
    "CLEANED_DIR=\"data/cleaned/\"\n",
    "\n",
    "Tag = namedtuple(\"Tag\", [\"token\", \"tag\"])\n",
    "\n",
    "def load_sequences(filename, sep=\"\\t\", notypes=False, test_data=False):\n",
    "    sequences = []\n",
    "    with open(filename) as fp:\n",
    "        seq = []\n",
    "        for line in fp:\n",
    "            line = line.strip()\n",
    "            if line:\n",
    "                line = line.split(sep)\n",
    "                if test_data:\n",
    "                    assert len(line) == 1\n",
    "                    line.append(\"?\")\n",
    "                if notypes:\n",
    "                    line[1] = line[1][0]\n",
    "                seq.append(Tag(*line))\n",
    "            else:\n",
    "                sequences.append(seq)\n",
    "                seq = []\n",
    "        if seq:\n",
    "            sequences.append(seq)\n",
    "    return sequences\n",
    "\n",
    "\n",
    "def load_vocab(filename):\n",
    "    vocab = set()\n",
    "    with open(filename) as fp:\n",
    "        for line in fp:\n",
    "            line = line.strip()\n",
    "            vocab.add(line)\n",
    "    return vocab      \n",
    "\n",
    "    \n",
    "def print_transitions(trans_features):\n",
    "    for (label_from, label_to), weight in trans_features:\n",
    "        print(\"%-6s -> %-7s %0.6f\" % (label_from, label_to, weight))\n",
    "        \n",
    "def print_state_features(state_features):\n",
    "    for (attr, label), weight in state_features:\n",
    "        print(\"%0.6f %-8s %s\" % (weight, label, attr)) \n",
    "        \n",
    "        \n",
    "def plot_cm(y_test, y_pred, labels=[], axis=1):\n",
    "    labels_s = dict((k,i) for i,k in enumerate(labels))\n",
    "    cm = np.zeros((len(labels), len(labels)))\n",
    "    for i,j in zip(sum(y_test, []), sum(y_pred, [])):\n",
    "        i = labels_s[i]\n",
    "        j = labels_s[j]\n",
    "        cm[i,j] += 1\n",
    "    with plt.rc_context(rc={'xtick.labelsize': 12, 'ytick.labelsize': 12,\n",
    "                       'figure.figsize': (16,14)}):\n",
    "        sns.heatmap(cm * 100/ cm.sum(axis=axis, keepdims=True),\n",
    "                    #cmap=sns.cubehelix_palette(n_colors=100, rot=-.4, as_cmap=True),\n",
    "                    cmap=\"Greys\",\n",
    "                    xticklabels=labels,\n",
    "                    yticklabels=labels)\n",
    "        plt.ylabel(\"True labels\")\n",
    "        plt.xlabel(\"Predicted labels\")\n",
    "        title = \"Precision Plot\"\n",
    "        if axis== 0:\n",
    "            title = \"Recall Plot\"\n",
    "        plt.title(title)\n",
    "    print cm.shape\n",
    "    return cm\n",
    "\n",
    "\n",
    "def print_sequences(sequences, predictions, filename, test_data=False, notypes=False):\n",
    "    with open(filename, \"wb+\") as fp:\n",
    "        for seq, pred in zip(sequences, predictions):\n",
    "            for t, p in zip(seq, pred):\n",
    "                token, tag = t\n",
    "                if tag[0] == \"U\":\n",
    "                    tag = \"B%s\" % tag[1:]\n",
    "                if tag[0] == \"E\":\n",
    "                    tag = \"I%s\" % tag[1:]\n",
    "                if p[0] == \"U\":\n",
    "                    p = \"B%s\" % p[1:]\n",
    "                if p[0] == \"E\":\n",
    "                    p = \"I%s\" % p[1:]\n",
    "                if notypes:\n",
    "                    tag = tag[0]\n",
    "                    p = p[0]\n",
    "                if test_data:\n",
    "                    line = \"\\t\".join((token, p))\n",
    "                else:\n",
    "                    line = \"\\t\".join((token, tag, p))\n",
    "                print >> fp, line\n",
    "            print >> fp, \"\"\n",
    "    print \"Done\"\n",
    "    \n",
    "    \n",
    "WORD_SPLITTER = re.compile(r'[\\p{Punct}\\s]+')\n",
    "class DictionaryFeatures:\n",
    "    def __init__(self, dictDir):\n",
    "        self.word2dictionaries = {}\n",
    "        self.word2hashtagdictionaries = {}\n",
    "        self.dictionaries = []\n",
    "        i = 0\n",
    "        for d in os.listdir(dictDir):\n",
    "            print >> sys.stderr, \"read dict %s\"%d\n",
    "            self.dictionaries.append(d)\n",
    "            if d == '.svn':\n",
    "                continue\n",
    "            for line in open(dictDir + \"/\" + d):\n",
    "                word = line.rstrip('\\n')\n",
    "                word = word.strip(' ').lower()\n",
    "                word_hashtag = \"\".join(WORD_SPLITTER.split(word))\n",
    "                if not self.word2dictionaries.has_key(word):\n",
    "                    self.word2dictionaries[word] = str(i)\n",
    "                else:   \n",
    "                    self.word2dictionaries[word] += \"\\t%s\" % i\n",
    "                if not self.word2hashtagdictionaries.has_key(word_hashtag):\n",
    "                    self.word2hashtagdictionaries[word_hashtag] = str(i)\n",
    "                else:\n",
    "                    self.word2hashtagdictionaries[word_hashtag] += \"\\t%s\" % i\n",
    "            i += 1\n",
    "    \n",
    "    MAX_WINDOW_SIZE=6\n",
    "    def GetDictFeatures(self, words, i):\n",
    "        features = []\n",
    "        for window in range(1,self.MAX_WINDOW_SIZE):\n",
    "            start=max(i-window+1, 0)\n",
    "            end = start + window\n",
    "            phrase = ' '.join(words[start:end]).lower().strip(string.punctuation)\n",
    "            if self.word2dictionaries.has_key(phrase):\n",
    "                for j in self.word2dictionaries[phrase].split('\\t'):\n",
    "                    features.append('DICT=%s' % self.dictionaries[int(j)])\n",
    "                if window > 1:\n",
    "                    features.append('DICTWIN=%s' % window)\n",
    "        return list(set(features))\n",
    "    \n",
    "    def GetHashtagDictFeatures(self, word):\n",
    "        features = []\n",
    "        if len(word) < 2 or word[0] != \"#\":\n",
    "            return features\n",
    "        word = word[1:].lower().strip(string.punctuation)\n",
    "        if self.word2hashtagdictionaries.has_key(word):\n",
    "            for j in self.word2hashtagdictionaries[word].split('\\t'):\n",
    "                features.append('DICT_HASHTAG=%s' % self.dictionaries[int(j)])\n",
    "        return list(set(features))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8023\n"
     ]
    }
   ],
   "source": [
    "train_sequences = load_sequences(\"data/cleaned/train.BIEOU.tsv\", sep=\"\\t\", notypes=False)\n",
    "dev_sequences = (load_sequences(\"data/cleaned/dev.BIEOU.tsv\", sep=\"\\t\", notypes=False) \n",
    "                 + load_sequences(\"data/cleaned/dev_2015.BIEOU.tsv\", sep=\"\\t\", notypes=False))\n",
    "\n",
    "vocab = load_vocab(\"vocab.no_extras.txt\")\n",
    "print len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "isHashtag 1002\n",
      "isMoney 8\n",
      "repeatedPunctuation 1388\n",
      "isURL 1159\n",
      "isMention 2243\n",
      "isNumber 393\n"
     ]
    }
   ],
   "source": [
    "all_sentences = [[t[0] for t in seq] for seq in (train_sequences+dev_sequences)]\n",
    "\n",
    "other_entities = {\n",
    "    \"isHashtag\": [],\n",
    "    \"isMention\": [],\n",
    "    \"isURL\": [],\n",
    "    \"isMoney\": [],\n",
    "    \"isNumber\": [],\n",
    "    \"repeatedPunctuation\": []\n",
    "}\n",
    "for seq in all_sentences:\n",
    "    for t in seq:\n",
    "        for k in other_entities.keys():\n",
    "            if RegexFeatures.PATTERNS[k].match(t):\n",
    "                other_entities[k].append(t)\n",
    "\n",
    "                \n",
    "for k, v in other_entities.iteritems():\n",
    "    print k, len(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'isHashtag': '__isHashtag__',\n",
       " 'isMention': '__isMention__',\n",
       " 'isMoney': '__isMoney__',\n",
       " 'isNumber': '__isNumber__',\n",
       " 'isURL': '__isURL__',\n",
       " 'repeatedPunctuation': '__repeatedPunctuation__'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ENTITY_MAPPINGS={k: \"__%s__\" % k for k in other_entities.keys()}\n",
    "ENTITY_MAPPINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess_token(x, to_lower=False):\n",
    "    for k in ENTITY_MAPPINGS.keys():\n",
    "        if RegexFeatures.PATTERNS[k].match(x):\n",
    "            return ENTITY_MAPPINGS[k]\n",
    "    if to_lower:\n",
    "        x = x.lower()\n",
    "    return x\n",
    "\n",
    "GLOVE_TWEET_MAPPINGS={\n",
    "    \"<user>\": \"isMention\",\n",
    "    \"<hashtag>\": \"isHashtag\",\n",
    "    \"<number>\": \"isDigit\",\n",
    "    \"<url>\": \"isURL\",\n",
    "    \"<allcaps>\": \"isAllCapitalWord\",\n",
    "}\n",
    "\n",
    "def process_glovevectors(filename):\n",
    "    words, dim = 0, 0\n",
    "    with open(filename) as fp:\n",
    "        for line in fp:\n",
    "            #line = line.strip().split(\" \")\n",
    "            words+= 1\n",
    "    line = line.strip().split(\" \")\n",
    "    dim = len(line) - 1\n",
    "    print \"Words: {}, dim: {}\".format(words, dim)\n",
    "    with open(filename) as fp, open(\"{}.processed.txt\".format(filename), \"wb+\") as fp1:\n",
    "        print >> fp1, words, dim\n",
    "        for line in fp:\n",
    "            line = line.strip().split(\" \", 1)\n",
    "            line[0] = dict.get(GLOVE_TWEET_MAPPINGS, line[0], line[0])\n",
    "            print >> fp1, line[0], line[1]\n",
    "    print \"Done\"\n",
    "\n",
    "    \n",
    "def tweet_features(sent, word2vec_model=None, cluster_vocabs=None, WORD_IDX=0):\n",
    "    features = {}\n",
    "    sent_length = len(sent) * 1.\n",
    "    for widx, word in enumerate(sent):\n",
    "        word = word[WORD_IDX]\n",
    "        lookup_key = preprocess_token(word, to_lower=True)\n",
    "        if word2vec_model and lookup_key in word2vec_model:\n",
    "            for i,v in enumerate(word2vec_model[lookup_key]):\n",
    "                features[\"_GLOBAL_WORDVEC_%s\" % i] = dict.get(features, \"_GLOBAL_WORDVEC_%s\" % i, 0) + v\n",
    "        if cluster_vocabs and lookup_key in cluster_vocabs:\n",
    "            v = dict.get(cluster_vocabs, lookup_key)\n",
    "            features[\"_GLOBAL_CLUSTER_=%s\" % v] = dict.get(features, \"_GLOBAL_CLUSTER_=%s\" % v, 0) + 1\n",
    "        if dict_features:\n",
    "            d_features = dict_features.GetDictFeatures([k[WORD_IDX] for k in sent], widx)\n",
    "            for k in d_features:\n",
    "                features[k] = dict.get(features, k, 0) + 1\n",
    "            d_hashtag_features = dict_features.GetHashtagDictFeatures(word)\n",
    "            for k in d_hashtag_features:\n",
    "                features[k] = dict.get(features, k, 0) + 1\n",
    "    #features = {k: v / sent_length for k,v in features.iteritems()}\n",
    "    return features\n",
    "\n",
    "def is_tweet_type(sent, cat_type):\n",
    "    for t in sent:\n",
    "        if t.tag != \"O\":\n",
    "            if t.tag[2:] == cat_type:\n",
    "                return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
